﻿using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace WebCrawler
{
    class Program
    {
        static HttpClient httpClient = new HttpClient();
        static HashSet<string> visitedUrls = new HashSet<string>();
        static object lockObj = new object();
        static List<string> phoneNumbers = new List<string>();

        static async Task Main(string[] args)
        {
            Console.Write("请输入要搜索的关键字：");
            string keyword = Console.ReadLine();
            string baiduUrl = $"https://www.baidu.com/s?wd={keyword}";
            string bingUrl = $"https://www.bing.com/search?q={keyword}";

            List<Task> tasks = new List<Task>()
            {
                Crawl(baiduUrl),
                Crawl(bingUrl)
            };

            await Task.WhenAll(tasks);

            Console.WriteLine("已爬取的URL：");
            foreach (var url in visitedUrls)
            {
                Console.WriteLine(url);
            }

            Console.WriteLine("100个有效的、不重复的高校电话号码：");
            foreach (var phoneNumber in phoneNumbers)
            {
                Console.WriteLine(phoneNumber);
            }
        }

        static async Task Crawl(string url)
        {
            try
            {
                var response = await httpClient.GetAsync(url);
                var content = await response.Content.ReadAsStringAsync();

                visitedUrls.Add(url);

                var phoneNumberRegex = new Regex(@"((\+86|86)?\s*\d{3,4}-\d{6,8})");

                var matches = phoneNumberRegex.Matches(content);

                foreach (Match match in matches)
                {
                    string phoneNumber = match.Value;
                    lock (lockObj)
                    {
                        if (!phoneNumbers.Contains(phoneNumber))
                        {
                            phoneNumbers.Add(phoneNumber);
                            Console.WriteLine($"从 {url} 中找到电话号码：{phoneNumber}");
                        }
                    }
                }

                var linkRegex = new Regex(@"<a.*?href=""(.*?)"".*?>(.*?)</a>", RegexOptions.IgnoreCase);
                var linkMatches = linkRegex.Matches(content);

                foreach (Match linkMatch in linkMatches)
                {
                    string linkUrl = linkMatch.Groups[1].Value;

                    if (linkUrl.StartsWith("http") && !visitedUrls.Contains(linkUrl))
                    {
                        await Crawl(linkUrl);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error crawling {url}: {ex.Message}");
            }
        }
    }
}
